# coding=utf-8
import requests
from parsel import Selector
from urllib.parse import urljoin
import threading
import time
import re
import os

os.mkdir(r"E:\爬虫项目\xbiquge小说爬取") if not os.path.exists(r"E:\爬虫项目\xbiquge小说爬取") else None
headers = {
    "user-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1 QQBrowser/6.9.11079.201"
}


def request(url):
    try:
        res = requests.get(url, headers=headers)
        r = bytes(res.text, res.encoding).decode('utf-8', errors='ignore')
        selector = Selector(text=r)
        return selector
    except Exception as e:
        print(e)


# 保存章节，sem为信号量，控制线程并发，减少网站压力
def save_chapter(chapter_url, sem):
    with sem:
        selector = request(chapter_url)
        chapter_name = selector.xpath("//h1/text()").get()
        parttern = re.compile('"|"|，|_|@|\*|\$|\||<|>|:|\?')  # 替换掉名称中的不支持Windows命令的特殊字符
        chapter_name = re.sub(parttern, '', chapter_name)
        content = "".join(selector.xpath("//div[@id='content']/text()").getall()).replace("\xa0", "")
        # 指定保存路径和保存名字
        with open(file=r"E:\爬虫项目\xbiquge小说爬取\%s.txt" % chapter_name, mode='w', encoding='utf-8') as f:
            f.write(content)
            print(f"{chapter_url + '-' + chapter_name}保存完毕......")


# 多线程实现
def run(url):
    sem = threading.Semaphore(50)  # 设置最大信号量,建议不超过200
    selector = request(url)
    chapters_url = selector.xpath("//div[@id='list']/dl/dd/a/@href").getall()
    tasks = [threading.Thread(target=save_chapter, args=(urljoin(url, chapter_url), sem)) for chapter_url in
             chapters_url]
    for t in tasks:
        t.start()
    for t in tasks:
        t.join()


if __name__ == '__main__':
    start = time.time()
    url = input("请输入需要爬取的小说详情页：").strip()
    run(url)
    end = time.time()
    print('爬虫运行结束，耗时: %.2fs' % (end - start))
